{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pprint\n",
    "import os\n",
    "import sys\n",
    "\n",
    "sys.path.append(os.path.dirname(os.getcwd()))\n",
    "from data import WN18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config:\n",
    "    seed = 21\n",
    "    n_epochs = 10\n",
    "    batch_size = 100\n",
    "    embed_dim = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "e: entity\n",
    "s: subject\n",
    "p: predicate\n",
    "o: object\n",
    "\"\"\"\n",
    "\n",
    "def read_triples(path):\n",
    "    triples = []\n",
    "    with open(path, 'rt') as f:\n",
    "        for line in f.readlines():\n",
    "            s, p, o = line.split()\n",
    "            triples += [(s.strip(), p.strip(), o.strip())]\n",
    "    return triples\n",
    "\n",
    "\n",
    "def load_triple():\n",
    "    WN18.download()\n",
    "    triples_tr = read_triples('../data/WN18/wn18/train.txt')\n",
    "    triples_va = read_triples('../data/WN18/wn18/valid.txt')\n",
    "    triples_te = read_triples('../data/WN18/wn18/test.txt')\n",
    "    \n",
    "    triples_all = triples_tr + triples_va + triples_te\n",
    "    \n",
    "    return triples_all, triples_tr, triples_va, triples_te\n",
    "\n",
    "\n",
    "def build_vocab(triples):\n",
    "    params = {}\n",
    "    \n",
    "    e_set = {s for (s, p, o) in triples} | {o for (s, p, o) in triples}\n",
    "    p_set = {p for (s, p, o) in triples}\n",
    "\n",
    "    params['e_vocab_size'] = len(e_set)\n",
    "    params['p_vocab_size'] = len(p_set)\n",
    "    params['embed_dim'] = Config.embed_dim\n",
    "\n",
    "    e2idx = {e: idx for idx, e in enumerate(sorted(e_set))}\n",
    "    p2idx = {p: idx for idx, p in enumerate(sorted(p_set))}\n",
    "    \n",
    "    return e2idx, p2idx, params\n",
    "\n",
    "\n",
    "def build_train_data(triples_tr, e2idx, p2idx):\n",
    "    x_s = np.array([e2idx[s] for (s, p, o) in triples_tr], dtype=np.int32)\n",
    "    x_p = np.array([p2idx[p] for (s, p, o) in triples_tr], dtype=np.int32)\n",
    "    x_o = np.array([e2idx[o] for (s, p, o) in triples_tr], dtype=np.int32)\n",
    "\n",
    "    x = {'s': x_s,\n",
    "         'p': x_p,\n",
    "         'o': x_o}\n",
    "    y = np.ones([len(x_s)], dtype=np.float32)\n",
    "    \n",
    "    return x, y\n",
    "\n",
    "\n",
    "def train_input_fn(triples_tr, e2idx, p2idx, random_state, params):\n",
    "    x, y = build_train_data(triples_tr, e2idx, p2idx)\n",
    "    s, p, o = x['s'], x['p'], x['o']\n",
    "    \n",
    "    s_ = random_state.choice(params['e_vocab_size'], s.shape)\n",
    "    o_ = random_state.choice(params['e_vocab_size'], o.shape)\n",
    "    \n",
    "    x_ = {\n",
    "        's': np.concatenate([s, s_, s]),\n",
    "        'p': np.concatenate([p, p, p]),\n",
    "        'o': np.concatenate([o, o, o_])}\n",
    "    y_ = np.concatenate([y, np.zeros([2*len(y)], dtype=np.float32)])\n",
    "    \n",
    "    return tf.estimator.inputs.numpy_input_fn(x = x_,\n",
    "                                              y = y_,\n",
    "                                              batch_size = Config.batch_size,\n",
    "                                              num_epochs = 1,\n",
    "                                              shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def s_next_batch(eval_triples,\n",
    "                 e2idx,\n",
    "                 p2idx,\n",
    "                 nb_entities,\n",
    "                 batch_size):\n",
    "    for (s, p, o) in tqdm(eval_triples, total=len(eval_triples), ncols=70):\n",
    "        s_idx, p_idx, o_idx = e2idx[s], p2idx[p], e2idx[o]\n",
    "        xs = np.arange(nb_entities)\n",
    "        xp = np.full(shape=(nb_entities,), fill_value=p_idx, dtype=np.int32)\n",
    "        xo = np.full(shape=(nb_entities,), fill_value=o_idx, dtype=np.int32)\n",
    "        for i in range(0, len(xs), batch_size):\n",
    "            yield xs[i: i+batch_size], xp[i: i+batch_size], xo[i: i+batch_size]\n",
    "\n",
    "\n",
    "def o_next_batch(eval_triples,\n",
    "                 e2idx, \n",
    "                 p2idx,\n",
    "                 nb_entities,\n",
    "                 batch_size):\n",
    "    for (s, p, o) in tqdm(eval_triples, total=len(eval_triples), ncols=70):\n",
    "        s_idx, p_idx, o_idx = e2idx[s], p2idx[p], e2idx[o]\n",
    "        xs = np.full(shape=(nb_entities,), fill_value=s_idx, dtype=np.int32)\n",
    "        xp = np.full(shape=(nb_entities,), fill_value=p_idx, dtype=np.int32)\n",
    "        xo = np.arange(nb_entities)\n",
    "        for i in range(0, len(xs), batch_size):\n",
    "            yield xs[i: i+batch_size], xp[i: i+batch_size], xo[i: i+batch_size]\n",
    "\n",
    "\n",
    "def s_input_fn(eval_triples,\n",
    "               e2idx, \n",
    "               p2idx,\n",
    "               nb_entities,\n",
    "               batch_size):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: s_next_batch(eval_triples,\n",
    "                             e2idx, \n",
    "                             p2idx,\n",
    "                             nb_entities,\n",
    "                             batch_size),\n",
    "        (tf.int32, tf.int32, tf.int32),\n",
    "        (tf.TensorShape([None,]),\n",
    "         tf.TensorShape([None,]),\n",
    "         tf.TensorShape([None,]),))\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    s, p, o = iterator.get_next()\n",
    "    return {'s': s, 'p': p, 'o': o}\n",
    "\n",
    "\n",
    "def o_input_fn(eval_triples,\n",
    "               e2idx, \n",
    "               p2idx,\n",
    "               nb_entities,\n",
    "               batch_size):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: o_next_batch(eval_triples,\n",
    "                             e2idx, \n",
    "                             p2idx,\n",
    "                             nb_entities,\n",
    "                             batch_size),\n",
    "        (tf.int32, tf.int32, tf.int32),\n",
    "        (tf.TensorShape([None,]),\n",
    "         tf.TensorShape([None,]),\n",
    "         tf.TensorShape([None,]),))\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    s, p, o = iterator.get_next()\n",
    "    return {'s': s, 'p': p, 'o': o}\n",
    "\n",
    "\n",
    "def evaluate_rank(model,\n",
    "                  valid_triples,\n",
    "                  test_triples,\n",
    "                  all_triples,\n",
    "                  e2idx,\n",
    "                  p2idx,\n",
    "                  nb_entities,\n",
    "                  batch_size):\n",
    "\n",
    "    #for eval_name, eval_triples in [('valid', valid_triples), ('test', test_triples)]:\n",
    "    for eval_name, eval_triples in [('test', test_triples)]:\n",
    "        \n",
    "        _scores_s = list(model.predict(\n",
    "            lambda: s_input_fn(eval_triples,\n",
    "                               e2idx, \n",
    "                               p2idx,\n",
    "                               nb_entities,\n",
    "                               batch_size), yield_single_examples=False))\n",
    "        \n",
    "        _scores_o = list(model.predict(\n",
    "            lambda: o_input_fn(eval_triples,\n",
    "                               e2idx, \n",
    "                               p2idx,\n",
    "                               nb_entities,\n",
    "                               batch_size), yield_single_examples=False))\n",
    "\n",
    "        ScoresS = np.concatenate(_scores_s).reshape([len(eval_triples), nb_entities])\n",
    "        ScoresO = np.concatenate(_scores_o).reshape([len(eval_triples), nb_entities])\n",
    "\n",
    "        ranks_s, ranks_o = [], []\n",
    "        filtered_ranks_s, filtered_ranks_o = [], []\n",
    "\n",
    "        for ((s, p, o), scores_s, scores_o) in tqdm(zip(eval_triples,\n",
    "                                                        ScoresS,\n",
    "                                                        ScoresO),\n",
    "                                                    total=len(eval_triples),\n",
    "                                                    ncols=70):\n",
    "            s_idx, p_idx, o_idx = e2idx[s], p2idx[p], e2idx[o]\n",
    "\n",
    "            ranks_s += [1 + np.argsort(np.argsort(- scores_s))[s_idx]]\n",
    "            ranks_o += [1 + np.argsort(np.argsort(- scores_o))[o_idx]]\n",
    "\n",
    "            filtered_scores_s = scores_s.copy()\n",
    "            filtered_scores_o = scores_o.copy()\n",
    "\n",
    "            rm_idx_s = [e2idx[fs] for (fs, fp, fo) in all_triples if fs != s and fp == p and fo == o]\n",
    "            rm_idx_o = [e2idx[fo] for (fs, fp, fo) in all_triples if fs == s and fp == p and fo != o]\n",
    "\n",
    "            filtered_scores_s[rm_idx_s] = - np.inf\n",
    "            filtered_scores_o[rm_idx_o] = - np.inf\n",
    "\n",
    "            filtered_ranks_s += [1 + np.argsort(np.argsort(- filtered_scores_s))[s_idx]]\n",
    "            filtered_ranks_o += [1 + np.argsort(np.argsort(- filtered_scores_o))[o_idx]]\n",
    "\n",
    "        ranks = ranks_s + ranks_o\n",
    "        filtered_ranks = filtered_ranks_s + filtered_ranks_o\n",
    "\n",
    "        for setting_name, setting_ranks in [('Raw', ranks), ('Filtered', filtered_ranks)]:\n",
    "            mean_rank = np.mean(1 / np.asarray(setting_ranks))\n",
    "            print('[{}] {} MRR: {}'.format(eval_name, setting_name, mean_rank))\n",
    "            for k in [1, 3, 5, 10]:\n",
    "                hits_at_k = np.mean(np.asarray(setting_ranks) <= k) * 100\n",
    "                print('[{}] {} Hits@{}: {}'.format(eval_name, setting_name, k, hits_at_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(features, params):\n",
    "    e_embed = tf.get_variable('e_embed',\n",
    "                              [params['e_vocab_size'], params['embed_dim']],\n",
    "                              initializer=tf.contrib.layers.xavier_initializer())\n",
    "    p_embed = tf.get_variable('p_embed',\n",
    "                              [params['p_vocab_size'], params['embed_dim']],\n",
    "                              initializer=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    s = tf.nn.embedding_lookup(e_embed, features['s'])\n",
    "    p = tf.nn.embedding_lookup(p_embed, features['p'])\n",
    "    o = tf.nn.embedding_lookup(e_embed, features['o'])\n",
    "    \n",
    "    logits = tf.reduce_sum(s * p * o, axis=1)\n",
    "    \n",
    "    return logits\n",
    "    \n",
    "    \n",
    "def model_fn(features, labels, mode, params):\n",
    "    logits = forward(features, params)\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        tf.logging.info('\\n'+pprint.pformat(tf.trainable_variables()))\n",
    "        tf.logging.info('params: %d'%count_train_params())\n",
    "        \n",
    "        loss_op = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits,\n",
    "                                                                         labels=labels))\n",
    "        \n",
    "        train_op = tf.train.AdamOptimizer().minimize(\n",
    "            loss_op, global_step=tf.train.get_global_step())\n",
    "        \n",
    "        return tf.estimator.EstimatorSpec(mode = mode,\n",
    "                                          loss = loss_op,\n",
    "                                          train_op = train_op)\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode, predictions = tf.sigmoid(logits))\n",
    "\n",
    "\n",
    "def count_train_params():\n",
    "    return np.sum([np.prod([d.value for d in v.get_shape()]) for v in tf.trainable_variables()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files Already Downloaded\n",
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x11cb85898>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.69314706, step = 1\n",
      "INFO:tensorflow:global_step/sec: 18.9888\n",
      "INFO:tensorflow:loss = 0.6931427, step = 101 (5.267 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.7144\n",
      "INFO:tensorflow:loss = 0.6931547, step = 201 (5.343 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.6406\n",
      "INFO:tensorflow:loss = 0.69316804, step = 301 (4.845 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.811\n",
      "INFO:tensorflow:loss = 0.69317216, step = 401 (4.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.983\n",
      "INFO:tensorflow:loss = 0.69317496, step = 501 (4.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.45\n",
      "INFO:tensorflow:loss = 0.69314, step = 601 (5.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5992\n",
      "INFO:tensorflow:loss = 0.69313055, step = 701 (5.376 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.128\n",
      "INFO:tensorflow:loss = 0.69314706, step = 801 (4.733 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.0572\n",
      "INFO:tensorflow:loss = 0.6931223, step = 901 (4.986 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.5753\n",
      "INFO:tensorflow:loss = 0.69308716, step = 1001 (4.430 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.3116\n",
      "INFO:tensorflow:loss = 0.6932614, step = 1101 (4.290 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6093\n",
      "INFO:tensorflow:loss = 0.6932356, step = 1201 (4.236 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.3161\n",
      "INFO:tensorflow:loss = 0.69315475, step = 1301 (4.691 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.003\n",
      "INFO:tensorflow:loss = 0.69299287, step = 1401 (4.545 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.9522\n",
      "INFO:tensorflow:loss = 0.69298697, step = 1501 (4.357 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.8578\n",
      "INFO:tensorflow:loss = 0.6929028, step = 1601 (4.375 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.7546\n",
      "INFO:tensorflow:loss = 0.69281137, step = 1701 (4.597 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.025\n",
      "INFO:tensorflow:loss = 0.6930126, step = 1801 (4.343 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1025\n",
      "INFO:tensorflow:loss = 0.69214284, step = 1901 (4.524 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.4788\n",
      "INFO:tensorflow:loss = 0.69260496, step = 2001 (4.449 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1127\n",
      "INFO:tensorflow:loss = 0.6921237, step = 2101 (4.522 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.8405\n",
      "INFO:tensorflow:loss = 0.69023615, step = 2201 (4.378 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.5703\n",
      "INFO:tensorflow:loss = 0.68840104, step = 2301 (4.431 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.5641\n",
      "INFO:tensorflow:loss = 0.68699175, step = 2401 (5.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.1088\n",
      "INFO:tensorflow:loss = 0.68333983, step = 2501 (4.737 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.9037\n",
      "INFO:tensorflow:loss = 0.7064549, step = 2601 (5.290 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.7993\n",
      "INFO:tensorflow:loss = 0.70010644, step = 2701 (4.808 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.2404\n",
      "INFO:tensorflow:loss = 0.6972245, step = 2801 (4.708 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.2649\n",
      "INFO:tensorflow:loss = 0.693871, step = 2901 (4.703 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.7874\n",
      "INFO:tensorflow:loss = 0.68916947, step = 3001 (5.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.2385\n",
      "INFO:tensorflow:loss = 0.6784719, step = 3101 (5.198 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.1323\n",
      "INFO:tensorflow:loss = 0.6666929, step = 3201 (5.836 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.6885\n",
      "INFO:tensorflow:loss = 0.64488006, step = 3301 (4.834 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.3963\n",
      "INFO:tensorflow:loss = 0.6317484, step = 3401 (4.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.1182\n",
      "INFO:tensorflow:loss = 0.6243284, step = 3501 (4.971 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.7762\n",
      "INFO:tensorflow:loss = 0.56164384, step = 3601 (5.057 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.657\n",
      "INFO:tensorflow:loss = 0.5439055, step = 3701 (5.360 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5712\n",
      "INFO:tensorflow:loss = 0.4992067, step = 3801 (5.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.2161\n",
      "INFO:tensorflow:loss = 0.44691327, step = 3901 (5.490 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.9683\n",
      "INFO:tensorflow:loss = 0.7002948, step = 4001 (5.272 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.3276\n",
      "INFO:tensorflow:loss = 0.6901908, step = 4101 (5.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5835\n",
      "INFO:tensorflow:loss = 0.6982782, step = 4201 (5.381 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 4244 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.67326236.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-4244\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 4245 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.709065, step = 4245\n",
      "INFO:tensorflow:global_step/sec: 17.1521\n",
      "INFO:tensorflow:loss = 0.69665164, step = 4345 (5.831 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.5177\n",
      "INFO:tensorflow:loss = 0.70792544, step = 4445 (5.709 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.8321\n",
      "INFO:tensorflow:loss = 0.7042872, step = 4545 (5.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.9424\n",
      "INFO:tensorflow:loss = 0.6843772, step = 4645 (5.279 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.0132\n",
      "INFO:tensorflow:loss = 0.6876523, step = 4745 (4.997 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.1047\n",
      "INFO:tensorflow:loss = 0.69178903, step = 4845 (5.234 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.1328\n",
      "INFO:tensorflow:loss = 0.71297485, step = 4945 (4.967 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.5401\n",
      "INFO:tensorflow:loss = 0.6923553, step = 5045 (5.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0732\n",
      "INFO:tensorflow:loss = 0.71533453, step = 5145 (5.533 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.8052\n",
      "INFO:tensorflow:loss = 0.68866724, step = 5245 (5.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.1174\n",
      "INFO:tensorflow:loss = 0.68766594, step = 5345 (5.231 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.9101\n",
      "INFO:tensorflow:loss = 0.68564343, step = 5445 (5.023 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5751\n",
      "INFO:tensorflow:loss = 0.69025666, step = 5545 (5.383 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.7413\n",
      "INFO:tensorflow:loss = 0.7098475, step = 5645 (5.066 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.5949\n",
      "INFO:tensorflow:loss = 0.6844662, step = 5745 (4.238 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6432\n",
      "INFO:tensorflow:loss = 0.69984686, step = 5845 (4.230 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.7293\n",
      "INFO:tensorflow:loss = 0.6905632, step = 5945 (4.214 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6728\n",
      "INFO:tensorflow:loss = 0.6923926, step = 6045 (4.224 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 23.3872\n",
      "INFO:tensorflow:loss = 0.6726109, step = 6145 (4.276 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.7174\n",
      "INFO:tensorflow:loss = 0.66494447, step = 6245 (4.216 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.4006\n",
      "INFO:tensorflow:loss = 0.7015714, step = 6345 (4.274 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6506\n",
      "INFO:tensorflow:loss = 0.64571726, step = 6445 (4.228 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.5666\n",
      "INFO:tensorflow:loss = 0.63766456, step = 6545 (4.243 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.1886\n",
      "INFO:tensorflow:loss = 0.63489205, step = 6645 (4.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.247\n",
      "INFO:tensorflow:loss = 0.60622185, step = 6745 (5.196 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.3756\n",
      "INFO:tensorflow:loss = 0.35748413, step = 6845 (5.442 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.9533\n",
      "INFO:tensorflow:loss = 0.2978887, step = 6945 (5.276 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.8434\n",
      "INFO:tensorflow:loss = 0.25057703, step = 7045 (5.307 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.5033\n",
      "INFO:tensorflow:loss = 0.24386992, step = 7145 (4.650 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1157\n",
      "INFO:tensorflow:loss = 0.17430353, step = 7245 (4.522 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.9753\n",
      "INFO:tensorflow:loss = 0.14777891, step = 7345 (4.551 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.8113\n",
      "INFO:tensorflow:loss = 0.15324564, step = 7445 (5.048 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.6663\n",
      "INFO:tensorflow:loss = 0.13069342, step = 7545 (6.000 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.3792\n",
      "INFO:tensorflow:loss = 0.09840252, step = 7645 (5.442 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5394\n",
      "INFO:tensorflow:loss = 0.10147356, step = 7745 (5.393 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.1487\n",
      "INFO:tensorflow:loss = 0.11141885, step = 7845 (6.193 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.5502\n",
      "INFO:tensorflow:loss = 0.07039122, step = 7945 (5.697 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.984\n",
      "INFO:tensorflow:loss = 0.061263923, step = 8045 (5.268 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0063\n",
      "INFO:tensorflow:loss = 0.06585812, step = 8145 (5.555 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.3062\n",
      "INFO:tensorflow:loss = 0.6820712, step = 8245 (5.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.1908\n",
      "INFO:tensorflow:loss = 0.68389654, step = 8345 (5.211 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0886\n",
      "INFO:tensorflow:loss = 0.6374679, step = 8445 (5.529 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 8488 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.65926677.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-8488\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 8489 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.70627713, step = 8489\n",
      "INFO:tensorflow:global_step/sec: 14.991\n",
      "INFO:tensorflow:loss = 0.69316745, step = 8589 (6.672 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.1001\n",
      "INFO:tensorflow:loss = 0.7123907, step = 8689 (5.236 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.0691\n",
      "INFO:tensorflow:loss = 0.6117647, step = 8789 (5.244 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.1273\n",
      "INFO:tensorflow:loss = 0.62967956, step = 8889 (5.229 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5816\n",
      "INFO:tensorflow:loss = 0.6040285, step = 8989 (5.381 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.6951\n",
      "INFO:tensorflow:loss = 0.5679525, step = 9089 (5.349 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.3568\n",
      "INFO:tensorflow:loss = 0.54813397, step = 9189 (5.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.0987\n",
      "INFO:tensorflow:loss = 0.55289495, step = 9289 (5.236 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.6944\n",
      "INFO:tensorflow:loss = 0.55439526, step = 9389 (5.349 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.5143\n",
      "INFO:tensorflow:loss = 0.5519141, step = 9489 (4.875 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.5233\n",
      "INFO:tensorflow:loss = 0.52285117, step = 9589 (4.873 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.0137\n",
      "INFO:tensorflow:loss = 0.52053785, step = 9689 (4.996 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.7606\n",
      "INFO:tensorflow:loss = 0.48222157, step = 9789 (5.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.7089\n",
      "INFO:tensorflow:loss = 0.48237976, step = 9889 (5.648 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.2529\n",
      "INFO:tensorflow:loss = 0.46810257, step = 9989 (5.478 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.1629\n",
      "INFO:tensorflow:loss = 0.4396764, step = 10089 (5.506 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5228\n",
      "INFO:tensorflow:loss = 0.39815006, step = 10189 (5.399 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.8834\n",
      "INFO:tensorflow:loss = 0.36463788, step = 10289 (5.923 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.911\n",
      "INFO:tensorflow:loss = 0.36528835, step = 10389 (5.583 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.4596\n",
      "INFO:tensorflow:loss = 0.35782278, step = 10489 (5.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.8059\n",
      "INFO:tensorflow:loss = 0.33015478, step = 10589 (5.049 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.9272\n",
      "INFO:tensorflow:loss = 0.33799803, step = 10689 (4.561 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.5662\n",
      "INFO:tensorflow:loss = 0.27436295, step = 10789 (5.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.0822\n",
      "INFO:tensorflow:loss = 0.29380134, step = 10889 (5.240 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.3777\n",
      "INFO:tensorflow:loss = 0.20853886, step = 10989 (5.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.7619\n",
      "INFO:tensorflow:loss = 0.14182463, step = 11089 (4.817 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.5927\n",
      "INFO:tensorflow:loss = 0.097617894, step = 11189 (4.856 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.6091\n",
      "INFO:tensorflow:loss = 0.07772133, step = 11289 (4.852 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.4462\n",
      "INFO:tensorflow:loss = 0.07148616, step = 11389 (5.421 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.0884\n",
      "INFO:tensorflow:loss = 0.06295708, step = 11489 (5.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.534\n",
      "INFO:tensorflow:loss = 0.06385079, step = 11589 (5.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.0756\n",
      "INFO:tensorflow:loss = 0.045847315, step = 11689 (5.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.0687\n",
      "INFO:tensorflow:loss = 0.029505847, step = 11789 (4.746 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.5291\n",
      "INFO:tensorflow:loss = 0.029596962, step = 11889 (4.871 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.8848\n",
      "INFO:tensorflow:loss = 0.032783784, step = 11989 (5.295 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.0877\n",
      "INFO:tensorflow:loss = 0.032949887, step = 12089 (4.979 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.724\n",
      "INFO:tensorflow:loss = 0.026687587, step = 12189 (5.340 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.5961\n",
      "INFO:tensorflow:loss = 0.021392925, step = 12289 (5.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.1246\n",
      "INFO:tensorflow:loss = 0.3425029, step = 12389 (5.229 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.0593\n",
      "INFO:tensorflow:loss = 0.3467485, step = 12489 (5.862 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.2399\n",
      "INFO:tensorflow:loss = 0.34921348, step = 12589 (5.197 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.1554\n",
      "INFO:tensorflow:loss = 0.38148364, step = 12689 (5.508 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 12732 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.3878312.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-12732\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saving checkpoints for 12733 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.01704003, step = 12733\n",
      "INFO:tensorflow:global_step/sec: 18.2516\n",
      "INFO:tensorflow:loss = 0.018545132, step = 12833 (5.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.5244\n",
      "INFO:tensorflow:loss = 0.017712018, step = 12933 (4.872 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.4456\n",
      "INFO:tensorflow:loss = 0.01871293, step = 13033 (4.663 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.6825\n",
      "INFO:tensorflow:loss = 0.015625024, step = 13133 (5.655 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.8977\n",
      "INFO:tensorflow:loss = 0.015269113, step = 13233 (5.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.3148\n",
      "INFO:tensorflow:loss = 0.31329942, step = 13333 (4.923 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.2266\n",
      "INFO:tensorflow:loss = 0.2799238, step = 13433 (4.499 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.8025\n",
      "INFO:tensorflow:loss = 0.29225558, step = 13533 (4.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.7829\n",
      "INFO:tensorflow:loss = 0.23080331, step = 13633 (4.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.9298\n",
      "INFO:tensorflow:loss = 0.24944304, step = 13733 (5.018 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.0468\n",
      "INFO:tensorflow:loss = 0.19819994, step = 13833 (5.250 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.8247\n",
      "INFO:tensorflow:loss = 0.21652916, step = 13933 (5.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.808\n",
      "INFO:tensorflow:loss = 0.19469698, step = 14033 (4.586 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.8303\n",
      "INFO:tensorflow:loss = 0.17974529, step = 14133 (4.581 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.0457\n",
      "INFO:tensorflow:loss = 0.14057483, step = 14233 (5.252 sec)\n",
      "INFO:tensorflow:global_step/sec: 15.727\n",
      "INFO:tensorflow:loss = 0.1541732, step = 14333 (6.357 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.229\n",
      "INFO:tensorflow:loss = 0.12753414, step = 14433 (5.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.6018\n",
      "INFO:tensorflow:loss = 0.135337, step = 14533 (4.629 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.2202\n",
      "INFO:tensorflow:loss = 0.1054274, step = 14633 (4.712 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.4245\n",
      "INFO:tensorflow:loss = 0.12603946, step = 14733 (4.668 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.0426\n",
      "INFO:tensorflow:loss = 0.109112814, step = 14833 (4.340 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.7484\n",
      "INFO:tensorflow:loss = 0.1035713, step = 14933 (4.598 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.4579\n",
      "INFO:tensorflow:loss = 0.07852591, step = 15033 (4.660 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.1862\n",
      "INFO:tensorflow:loss = 0.08772448, step = 15133 (4.720 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.5746\n",
      "INFO:tensorflow:loss = 0.06528525, step = 15233 (4.430 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0829\n",
      "INFO:tensorflow:loss = 0.06546411, step = 15333 (5.531 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.2261\n",
      "INFO:tensorflow:loss = 0.053861313, step = 15433 (5.486 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.9537\n",
      "INFO:tensorflow:loss = 0.049342174, step = 15533 (5.012 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.1684\n",
      "INFO:tensorflow:loss = 0.054858297, step = 15633 (4.958 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.3639\n",
      "INFO:tensorflow:loss = 0.055179473, step = 15733 (4.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.0099\n",
      "INFO:tensorflow:loss = 0.040231153, step = 15833 (4.543 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.4423\n",
      "INFO:tensorflow:loss = 0.042134915, step = 15933 (4.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.5065\n",
      "INFO:tensorflow:loss = 0.050967887, step = 16033 (4.650 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.0493\n",
      "INFO:tensorflow:loss = 0.3055373, step = 16133 (4.339 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.2329\n",
      "INFO:tensorflow:loss = 0.10320681, step = 16233 (4.304 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.5848\n",
      "INFO:tensorflow:loss = 0.10581257, step = 16333 (4.428 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.0113\n",
      "INFO:tensorflow:loss = 0.048511602, step = 16433 (4.345 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6786\n",
      "INFO:tensorflow:loss = 0.043444604, step = 16533 (4.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.8782\n",
      "INFO:tensorflow:loss = 0.027671626, step = 16633 (4.571 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.4548\n",
      "INFO:tensorflow:loss = 0.024689138, step = 16733 (4.264 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.565\n",
      "INFO:tensorflow:loss = 0.021056812, step = 16833 (4.244 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.4938\n",
      "INFO:tensorflow:loss = 0.014035736, step = 16933 (4.256 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 16976 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.007913809.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-16976\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 16977 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0060514687, step = 16977\n",
      "INFO:tensorflow:global_step/sec: 21.2775\n",
      "INFO:tensorflow:loss = 0.0063307025, step = 17077 (4.701 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.5885\n",
      "INFO:tensorflow:loss = 0.00802611, step = 17177 (4.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.8377\n",
      "INFO:tensorflow:loss = 0.0069016023, step = 17277 (5.309 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.2519\n",
      "INFO:tensorflow:loss = 0.011692405, step = 17377 (5.479 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.5455\n",
      "INFO:tensorflow:loss = 0.006063161, step = 17477 (5.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.8027\n",
      "INFO:tensorflow:loss = 0.0050212555, step = 17577 (4.386 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.9373\n",
      "INFO:tensorflow:loss = 0.06863259, step = 17677 (5.576 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0266\n",
      "INFO:tensorflow:loss = 0.10005045, step = 17777 (5.547 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.6593\n",
      "INFO:tensorflow:loss = 0.0865725, step = 17877 (5.358 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.6337\n",
      "INFO:tensorflow:loss = 0.06057509, step = 17977 (4.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.5208\n",
      "INFO:tensorflow:loss = 0.08184672, step = 18077 (4.647 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.6867\n",
      "INFO:tensorflow:loss = 0.06300704, step = 18177 (4.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.1089\n",
      "INFO:tensorflow:loss = 0.04228496, step = 18277 (4.973 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.241\n",
      "INFO:tensorflow:loss = 0.05065184, step = 18377 (4.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.1115\n",
      "INFO:tensorflow:loss = 0.047400657, step = 18477 (4.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.4813\n",
      "INFO:tensorflow:loss = 0.042379532, step = 18577 (4.448 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.5703\n",
      "INFO:tensorflow:loss = 0.054841008, step = 18677 (4.431 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.8594\n",
      "INFO:tensorflow:loss = 0.031451304, step = 18777 (4.375 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.7037\n",
      "INFO:tensorflow:loss = 0.03365436, step = 18877 (4.404 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.0389\n",
      "INFO:tensorflow:loss = 0.03218161, step = 18977 (4.753 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.9809\n",
      "INFO:tensorflow:loss = 0.02342649, step = 19077 (4.351 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.2881\n",
      "INFO:tensorflow:loss = 0.036199193, step = 19177 (4.698 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.0744\n",
      "INFO:tensorflow:loss = 0.022037653, step = 19277 (4.530 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.2241\n",
      "INFO:tensorflow:loss = 0.034433387, step = 19377 (4.306 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.2634\n",
      "INFO:tensorflow:loss = 0.01901976, step = 19477 (5.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.2018\n",
      "INFO:tensorflow:loss = 0.022837825, step = 19577 (4.950 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0019\n",
      "INFO:tensorflow:loss = 0.0215137, step = 19677 (5.555 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.7586\n",
      "INFO:tensorflow:loss = 0.018941345, step = 19777 (5.061 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.9851\n",
      "INFO:tensorflow:loss = 0.063112564, step = 19877 (5.268 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 21.6629\n",
      "INFO:tensorflow:loss = 0.014801575, step = 19977 (4.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.3141\n",
      "INFO:tensorflow:loss = 0.012151762, step = 20077 (4.289 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1533\n",
      "INFO:tensorflow:loss = 0.025867328, step = 20177 (4.514 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.2846\n",
      "INFO:tensorflow:loss = 0.00982687, step = 20277 (4.698 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.8553\n",
      "INFO:tensorflow:loss = 0.017311504, step = 20377 (5.304 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.2244\n",
      "INFO:tensorflow:loss = 0.009361813, step = 20477 (5.806 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.6642\n",
      "INFO:tensorflow:loss = 0.032713365, step = 20577 (5.358 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.7069\n",
      "INFO:tensorflow:loss = 0.03641712, step = 20677 (5.647 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.3619\n",
      "INFO:tensorflow:loss = 0.022188032, step = 20777 (4.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.7011\n",
      "INFO:tensorflow:loss = 0.013359409, step = 20877 (4.831 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.8073\n",
      "INFO:tensorflow:loss = 0.00841957, step = 20977 (5.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.6532\n",
      "INFO:tensorflow:loss = 0.0077739377, step = 21077 (4.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.8722\n",
      "INFO:tensorflow:loss = 0.006789403, step = 21177 (5.032 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 21220 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0057440926.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-21220\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 21221 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.027551975, step = 21221\n",
      "INFO:tensorflow:global_step/sec: 18.6134\n",
      "INFO:tensorflow:loss = 0.028777516, step = 21321 (5.374 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.3555\n",
      "INFO:tensorflow:loss = 0.027254479, step = 21421 (5.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.9208\n",
      "INFO:tensorflow:loss = 0.014906948, step = 21521 (4.780 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.4995\n",
      "INFO:tensorflow:loss = 0.018657189, step = 21621 (4.878 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.4938\n",
      "INFO:tensorflow:loss = 0.021550264, step = 21721 (4.879 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.0948\n",
      "INFO:tensorflow:loss = 0.015113223, step = 21821 (4.976 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.0682\n",
      "INFO:tensorflow:loss = 0.021688139, step = 21921 (4.983 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.4233\n",
      "INFO:tensorflow:loss = 0.018495696, step = 22021 (4.668 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1506\n",
      "INFO:tensorflow:loss = 0.061711453, step = 22121 (4.515 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.0536\n",
      "INFO:tensorflow:loss = 0.012724804, step = 22221 (4.338 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.6815\n",
      "INFO:tensorflow:loss = 0.011280216, step = 22321 (4.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.4129\n",
      "INFO:tensorflow:loss = 0.0072739245, step = 22421 (4.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.5232\n",
      "INFO:tensorflow:loss = 0.0048406566, step = 22521 (5.707 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.7515\n",
      "INFO:tensorflow:loss = 0.00808547, step = 22621 (5.970 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.5186\n",
      "INFO:tensorflow:loss = 0.007929973, step = 22721 (6.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.158\n",
      "INFO:tensorflow:loss = 0.005849272, step = 22821 (5.828 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5678\n",
      "INFO:tensorflow:loss = 0.003511674, step = 22921 (5.387 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.2305\n",
      "INFO:tensorflow:loss = 0.010407239, step = 23021 (5.199 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.4026\n",
      "INFO:tensorflow:loss = 0.0061097033, step = 23121 (5.434 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.8434\n",
      "INFO:tensorflow:loss = 0.0050098146, step = 23221 (4.378 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.8428\n",
      "INFO:tensorflow:loss = 0.006763211, step = 23321 (4.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.7882\n",
      "INFO:tensorflow:loss = 0.005013268, step = 23421 (4.590 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.3463\n",
      "INFO:tensorflow:loss = 0.0043582595, step = 23521 (5.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.6633\n",
      "INFO:tensorflow:loss = 0.0027937605, step = 23621 (4.839 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.7311\n",
      "INFO:tensorflow:loss = 0.0035937165, step = 23721 (4.399 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.9655\n",
      "INFO:tensorflow:loss = 0.007447202, step = 23821 (4.553 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.3662\n",
      "INFO:tensorflow:loss = 0.0063921646, step = 23921 (4.471 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.9929\n",
      "INFO:tensorflow:loss = 0.0036727532, step = 24021 (4.349 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1616\n",
      "INFO:tensorflow:loss = 0.002183601, step = 24121 (4.512 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.6835\n",
      "INFO:tensorflow:loss = 0.001325385, step = 24221 (4.409 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.7725\n",
      "INFO:tensorflow:loss = 0.0007612219, step = 24321 (4.391 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.1097\n",
      "INFO:tensorflow:loss = 0.00076289073, step = 24421 (4.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6035\n",
      "INFO:tensorflow:loss = 0.027620045, step = 24521 (4.236 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.3793\n",
      "INFO:tensorflow:loss = 0.011541442, step = 24621 (4.277 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.8599\n",
      "INFO:tensorflow:loss = 0.016954057, step = 24721 (4.375 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.2633\n",
      "INFO:tensorflow:loss = 0.011745892, step = 24821 (4.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.028\n",
      "INFO:tensorflow:loss = 0.005891394, step = 24921 (4.540 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.7244\n",
      "INFO:tensorflow:loss = 0.0048441524, step = 25021 (4.603 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.6204\n",
      "INFO:tensorflow:loss = 0.04852784, step = 25121 (4.421 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.8469\n",
      "INFO:tensorflow:loss = 0.00586306, step = 25221 (4.377 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.4961\n",
      "INFO:tensorflow:loss = 0.0061402405, step = 25321 (4.256 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1398\n",
      "INFO:tensorflow:loss = 0.10533691, step = 25421 (4.516 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 25464 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.004540586.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-25464\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 25465 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0056049484, step = 25465\n",
      "INFO:tensorflow:global_step/sec: 20.4231\n",
      "INFO:tensorflow:loss = 0.0052334797, step = 25565 (4.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.1385\n",
      "INFO:tensorflow:loss = 0.0061073136, step = 25665 (5.512 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.0533\n",
      "INFO:tensorflow:loss = 0.0052173375, step = 25765 (4.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.6623\n",
      "INFO:tensorflow:loss = 0.004946339, step = 25865 (5.086 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.6188\n",
      "INFO:tensorflow:loss = 0.0040215296, step = 25965 (4.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.4112\n",
      "INFO:tensorflow:loss = 0.0032685418, step = 26065 (4.271 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.0343\n",
      "INFO:tensorflow:loss = 0.0040775384, step = 26165 (4.341 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.3022\n",
      "INFO:tensorflow:loss = 0.0041862675, step = 26265 (4.291 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 23.1391\n",
      "INFO:tensorflow:loss = 0.0012555821, step = 26365 (4.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.2662\n",
      "INFO:tensorflow:loss = 0.0017564258, step = 26465 (4.298 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.1823\n",
      "INFO:tensorflow:loss = 0.042840205, step = 26565 (4.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.0005\n",
      "INFO:tensorflow:loss = 0.0024591233, step = 26665 (4.348 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.3357\n",
      "INFO:tensorflow:loss = 0.0022110469, step = 26765 (4.285 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.1628\n",
      "INFO:tensorflow:loss = 0.0018130823, step = 26865 (4.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.2201\n",
      "INFO:tensorflow:loss = 0.0019714388, step = 26965 (4.306 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.8812\n",
      "INFO:tensorflow:loss = 0.0014880953, step = 27065 (4.789 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.5452\n",
      "INFO:tensorflow:loss = 0.002382321, step = 27165 (4.641 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.7355\n",
      "INFO:tensorflow:loss = 0.0011084648, step = 27265 (4.601 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.4755\n",
      "INFO:tensorflow:loss = 0.00095165014, step = 27365 (4.656 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.346\n",
      "INFO:tensorflow:loss = 0.00084718823, step = 27465 (4.475 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.0165\n",
      "INFO:tensorflow:loss = 0.0015295739, step = 27565 (4.345 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.2895\n",
      "INFO:tensorflow:loss = 0.0007974436, step = 27665 (4.487 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.083\n",
      "INFO:tensorflow:loss = 0.0003735877, step = 27765 (4.528 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.0179\n",
      "INFO:tensorflow:loss = 0.009210809, step = 27865 (4.542 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.526\n",
      "INFO:tensorflow:loss = 0.0003234796, step = 27965 (4.439 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.3631\n",
      "INFO:tensorflow:loss = 0.007525563, step = 28065 (4.280 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.1696\n",
      "INFO:tensorflow:loss = 0.0036041155, step = 28165 (4.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.5004\n",
      "INFO:tensorflow:loss = 0.0058914702, step = 28265 (4.444 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.4845\n",
      "INFO:tensorflow:loss = 0.0026205897, step = 28365 (4.448 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.9856\n",
      "INFO:tensorflow:loss = 0.0024833437, step = 28465 (4.350 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.1026\n",
      "INFO:tensorflow:loss = 0.0036368342, step = 28565 (4.739 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.2745\n",
      "INFO:tensorflow:loss = 0.0024192624, step = 28665 (5.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.3357\n",
      "INFO:tensorflow:loss = 0.0031171325, step = 28765 (5.454 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.8583\n",
      "INFO:tensorflow:loss = 0.0025598148, step = 28865 (5.600 sec)\n",
      "INFO:tensorflow:global_step/sec: 15.4721\n",
      "INFO:tensorflow:loss = 0.0031421962, step = 28965 (6.463 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.7709\n",
      "INFO:tensorflow:loss = 0.0014092599, step = 29065 (5.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.1866\n",
      "INFO:tensorflow:loss = 0.0022933695, step = 29165 (5.212 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.1177\n",
      "INFO:tensorflow:loss = 0.0024425534, step = 29265 (5.521 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.3176\n",
      "INFO:tensorflow:loss = 0.00065513724, step = 29365 (6.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.7031\n",
      "INFO:tensorflow:loss = 0.0010613801, step = 29465 (5.347 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.0154\n",
      "INFO:tensorflow:loss = 0.0032358859, step = 29565 (4.542 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.5613\n",
      "INFO:tensorflow:loss = 0.06818215, step = 29665 (4.432 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 29708 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.00094780116.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-29708\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 29709 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.00083640474, step = 29709\n",
      "INFO:tensorflow:global_step/sec: 18.0188\n",
      "INFO:tensorflow:loss = 0.00051895884, step = 29809 (5.551 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.835\n",
      "INFO:tensorflow:loss = 0.0014080704, step = 29909 (5.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.1817\n",
      "INFO:tensorflow:loss = 0.001492296, step = 30009 (6.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.7348\n",
      "INFO:tensorflow:loss = 0.0026579166, step = 30109 (5.337 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5453\n",
      "INFO:tensorflow:loss = 0.0007277533, step = 30209 (5.392 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.8307\n",
      "INFO:tensorflow:loss = 0.0004915748, step = 30309 (5.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.2814\n",
      "INFO:tensorflow:loss = 0.022407874, step = 30409 (5.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.6392\n",
      "INFO:tensorflow:loss = 0.0005633314, step = 30509 (4.845 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.4139\n",
      "INFO:tensorflow:loss = 0.00053537346, step = 30609 (4.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.4322\n",
      "INFO:tensorflow:loss = 0.001497912, step = 30709 (4.894 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.5857\n",
      "INFO:tensorflow:loss = 0.00037449025, step = 30809 (4.240 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.7164\n",
      "INFO:tensorflow:loss = 0.0005342405, step = 30909 (4.605 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.3371\n",
      "INFO:tensorflow:loss = 0.0006638422, step = 31009 (4.687 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.7838\n",
      "INFO:tensorflow:loss = 0.0003377067, step = 31109 (4.389 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.8125\n",
      "INFO:tensorflow:loss = 0.00060354685, step = 31209 (4.584 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.4053\n",
      "INFO:tensorflow:loss = 0.00037935175, step = 31309 (4.672 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.9496\n",
      "INFO:tensorflow:loss = 0.00029624542, step = 31409 (5.277 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.9672\n",
      "INFO:tensorflow:loss = 0.0002546609, step = 31509 (5.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.9136\n",
      "INFO:tensorflow:loss = 0.0009843615, step = 31609 (4.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.4804\n",
      "INFO:tensorflow:loss = 0.0004252024, step = 31709 (5.412 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.78\n",
      "INFO:tensorflow:loss = 0.040201653, step = 31809 (5.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.8773\n",
      "INFO:tensorflow:loss = 0.00053586735, step = 31909 (5.297 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.8426\n",
      "INFO:tensorflow:loss = 0.00053928304, step = 32009 (4.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.6705\n",
      "INFO:tensorflow:loss = 0.00051251036, step = 32109 (5.660 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.004\n",
      "INFO:tensorflow:loss = 0.046249803, step = 32209 (5.880 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.2264\n",
      "INFO:tensorflow:loss = 0.00050219236, step = 32309 (4.499 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.9812\n",
      "INFO:tensorflow:loss = 0.013046818, step = 32409 (4.549 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.8599\n",
      "INFO:tensorflow:loss = 0.0040697358, step = 32509 (4.575 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.2011\n",
      "INFO:tensorflow:loss = 0.0031304138, step = 32609 (4.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.3481\n",
      "INFO:tensorflow:loss = 0.001562402, step = 32709 (4.475 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.0221\n",
      "INFO:tensorflow:loss = 0.0013700275, step = 32809 (5.257 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.5004\n",
      "INFO:tensorflow:loss = 0.0009665063, step = 32909 (7.407 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.0524\n",
      "INFO:tensorflow:loss = 0.003551838, step = 33009 (7.661 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.7598\n",
      "INFO:tensorflow:loss = 0.0010213587, step = 33109 (5.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.5257\n",
      "INFO:tensorflow:loss = 0.00040668165, step = 33209 (5.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.6792\n",
      "INFO:tensorflow:loss = 0.0004595594, step = 33309 (4.836 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.723\n",
      "INFO:tensorflow:loss = 0.00030957576, step = 33409 (5.070 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.9558\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.0001654242, step = 33509 (5.275 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.0391\n",
      "INFO:tensorflow:loss = 0.0003932778, step = 33609 (5.869 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0545\n",
      "INFO:tensorflow:loss = 7.2048475e-05, step = 33709 (5.539 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.2269\n",
      "INFO:tensorflow:loss = 0.00097653596, step = 33809 (5.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.798\n",
      "INFO:tensorflow:loss = 0.0066415113, step = 33909 (5.320 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 33952 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0053888108.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-33952\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 33953 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.002073142, step = 33953\n",
      "INFO:tensorflow:global_step/sec: 19.8461\n",
      "INFO:tensorflow:loss = 0.0026106646, step = 34053 (5.040 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.4649\n",
      "INFO:tensorflow:loss = 0.0014083182, step = 34153 (4.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.9584\n",
      "INFO:tensorflow:loss = 0.0010592415, step = 34253 (7.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.5136\n",
      "INFO:tensorflow:loss = 0.0017040473, step = 34353 (7.399 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.9153\n",
      "INFO:tensorflow:loss = 0.00070300064, step = 34453 (4.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.3963\n",
      "INFO:tensorflow:loss = 0.0020592741, step = 34553 (5.748 sec)\n",
      "INFO:tensorflow:global_step/sec: 14.9069\n",
      "INFO:tensorflow:loss = 0.0008578468, step = 34653 (6.709 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.6995\n",
      "INFO:tensorflow:loss = 0.00047053274, step = 34753 (5.650 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.0212\n",
      "INFO:tensorflow:loss = 0.07006449, step = 34853 (6.242 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.9203\n",
      "INFO:tensorflow:loss = 0.00044210884, step = 34953 (5.580 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.138\n",
      "INFO:tensorflow:loss = 0.00046900695, step = 35053 (5.837 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.0877\n",
      "INFO:tensorflow:loss = 0.001002129, step = 35153 (5.851 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.4833\n",
      "INFO:tensorflow:loss = 0.00040382665, step = 35253 (5.719 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.2462\n",
      "INFO:tensorflow:loss = 0.00043317134, step = 35353 (4.707 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.0412\n",
      "INFO:tensorflow:loss = 0.00010648229, step = 35453 (4.752 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.629\n",
      "INFO:tensorflow:loss = 0.00031984207, step = 35553 (4.232 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6172\n",
      "INFO:tensorflow:loss = 0.00020387206, step = 35653 (4.234 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.5554\n",
      "INFO:tensorflow:loss = 0.00021332892, step = 35753 (4.245 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6367\n",
      "INFO:tensorflow:loss = 0.000150941, step = 35853 (4.231 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.665\n",
      "INFO:tensorflow:loss = 0.00018798928, step = 35953 (4.226 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.7074\n",
      "INFO:tensorflow:loss = 0.00020020583, step = 36053 (4.218 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6024\n",
      "INFO:tensorflow:loss = 0.00021925654, step = 36153 (4.237 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.7913\n",
      "INFO:tensorflow:loss = 0.00014449314, step = 36253 (4.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.6163\n",
      "INFO:tensorflow:loss = 0.00012861005, step = 36353 (4.234 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.4909\n",
      "INFO:tensorflow:loss = 0.00014291776, step = 36453 (4.257 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.5534\n",
      "INFO:tensorflow:loss = 0.001211634, step = 36553 (4.246 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.5895\n",
      "INFO:tensorflow:loss = 9.6958254e-05, step = 36653 (4.427 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.0365\n",
      "INFO:tensorflow:loss = 4.5261444e-05, step = 36753 (4.754 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.011\n",
      "INFO:tensorflow:loss = 0.00022189932, step = 36853 (4.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.2591\n",
      "INFO:tensorflow:loss = 0.001379354, step = 36953 (4.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.0451\n",
      "INFO:tensorflow:loss = 0.0004290728, step = 37053 (4.536 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.4669\n",
      "INFO:tensorflow:loss = 0.00030985137, step = 37153 (4.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.3092\n",
      "INFO:tensorflow:loss = 0.002211073, step = 37253 (4.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.1973\n",
      "INFO:tensorflow:loss = 0.0004180925, step = 37353 (5.209 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.9333\n",
      "INFO:tensorflow:loss = 0.0007346885, step = 37453 (5.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.7131\n",
      "INFO:tensorflow:loss = 0.0002238658, step = 37553 (5.983 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.8957\n",
      "INFO:tensorflow:loss = 0.00023342339, step = 37653 (5.589 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.1526\n",
      "INFO:tensorflow:loss = 0.00024780037, step = 37753 (6.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.6135\n",
      "INFO:tensorflow:loss = 0.00030299567, step = 37853 (5.678 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.4601\n",
      "INFO:tensorflow:loss = 0.00044184513, step = 37953 (5.727 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.5569\n",
      "INFO:tensorflow:loss = 0.00014252235, step = 38053 (5.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.7506\n",
      "INFO:tensorflow:loss = 0.00026665203, step = 38153 (5.063 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 38196 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.00014221386.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 8192200\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-38196\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 38197 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.00033769716, step = 38197\n",
      "INFO:tensorflow:global_step/sec: 22.0875\n",
      "INFO:tensorflow:loss = 0.0008036875, step = 38297 (4.529 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.3141\n",
      "INFO:tensorflow:loss = 0.0002456949, step = 38397 (4.289 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.6562\n",
      "INFO:tensorflow:loss = 0.00019973717, step = 38497 (4.414 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.8103\n",
      "INFO:tensorflow:loss = 0.00020593802, step = 38597 (4.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.3459\n",
      "INFO:tensorflow:loss = 0.00015186504, step = 38697 (4.476 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.7756\n",
      "INFO:tensorflow:loss = 7.5090975e-05, step = 38797 (4.390 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.0594\n",
      "INFO:tensorflow:loss = 7.047971e-05, step = 38897 (4.533 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.0433\n",
      "INFO:tensorflow:loss = 7.063349e-05, step = 38997 (4.536 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.7445\n",
      "INFO:tensorflow:loss = 0.0058986824, step = 39097 (4.397 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.5365\n",
      "INFO:tensorflow:loss = 0.00022165125, step = 39197 (5.395 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.1548\n",
      "INFO:tensorflow:loss = 0.0003220124, step = 39297 (5.508 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.9848\n",
      "INFO:tensorflow:loss = 0.0004905303, step = 39397 (5.268 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.3045\n",
      "INFO:tensorflow:loss = 0.0063475235, step = 39497 (5.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.5313\n",
      "INFO:tensorflow:loss = 0.00022775595, step = 39597 (5.704 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.3096\n",
      "INFO:tensorflow:loss = 0.00014661223, step = 39697 (6.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.1207\n",
      "INFO:tensorflow:loss = 0.016439062, step = 39797 (5.839 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 17.473\n",
      "INFO:tensorflow:loss = 0.0003257164, step = 39897 (5.722 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.2158\n",
      "INFO:tensorflow:loss = 0.00014598526, step = 39997 (5.490 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.3956\n",
      "INFO:tensorflow:loss = 0.00010825575, step = 40097 (5.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.9661\n",
      "INFO:tensorflow:loss = 0.00070927583, step = 40197 (5.566 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.0496\n",
      "INFO:tensorflow:loss = 0.0034223069, step = 40297 (5.249 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0324\n",
      "INFO:tensorflow:loss = 0.00042293247, step = 40397 (5.545 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.5049\n",
      "INFO:tensorflow:loss = 0.00044064978, step = 40497 (5.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.3157\n",
      "INFO:tensorflow:loss = 0.0017988888, step = 40597 (4.922 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.4485\n",
      "INFO:tensorflow:loss = 0.06859967, step = 40697 (4.265 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.1688\n",
      "INFO:tensorflow:loss = 0.00037308742, step = 40797 (4.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.3103\n",
      "INFO:tensorflow:loss = 0.00017299224, step = 40897 (4.290 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.562\n",
      "INFO:tensorflow:loss = 7.7521014e-05, step = 40997 (4.432 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.1975\n",
      "INFO:tensorflow:loss = 0.00017318588, step = 41097 (4.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.3218\n",
      "INFO:tensorflow:loss = 0.00019936844, step = 41197 (4.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.8181\n",
      "INFO:tensorflow:loss = 7.2876166e-05, step = 41297 (4.584 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.1141\n",
      "INFO:tensorflow:loss = 7.273625e-05, step = 41397 (4.736 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.587\n",
      "INFO:tensorflow:loss = 8.798056e-05, step = 41497 (4.427 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.9001\n",
      "INFO:tensorflow:loss = 8.9287496e-05, step = 41597 (4.367 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1358\n",
      "INFO:tensorflow:loss = 0.00013786685, step = 41697 (4.518 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.5919\n",
      "INFO:tensorflow:loss = 9.8516626e-05, step = 41797 (4.856 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.6982\n",
      "INFO:tensorflow:loss = 7.046861e-05, step = 41897 (4.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.8763\n",
      "INFO:tensorflow:loss = 0.0014168008, step = 41997 (4.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.828\n",
      "INFO:tensorflow:loss = 0.0016385274, step = 42097 (4.801 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.2842\n",
      "INFO:tensorflow:loss = 0.0006422888, step = 42197 (5.469 sec)\n",
      "INFO:tensorflow:global_step/sec: 21.6164\n",
      "INFO:tensorflow:loss = 0.00014678959, step = 42297 (4.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 22.1754\n",
      "INFO:tensorflow:loss = 0.00020851778, step = 42397 (4.509 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 42440 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.00026877964.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-42440\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████| 5000/5000 [02:16<00:00, 36.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph9gee3t2/model.ckpt-42440\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████| 5000/5000 [02:20<00:00, 35.56it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [04:02<00:00, 20.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.5655667568354295\n",
      "[test] Raw Hits@1: 42.699999999999996\n",
      "[test] Raw Hits@3: 66.03999999999999\n",
      "[test] Raw Hits@5: 74.53999999999999\n",
      "[test] Raw Hits@10: 81.76\n",
      "[test] Filtered MRR: 0.8155054696611849\n",
      "[test] Filtered Hits@1: 70.89\n",
      "[test] Filtered Hits@3: 92.0\n",
      "[test] Filtered Hits@5: 93.33\n",
      "[test] Filtered Hits@10: 94.16\n"
     ]
    }
   ],
   "source": [
    "random_state = np.random.RandomState(Config.seed)\n",
    "triples_all, triples_tr, triples_va, triples_te = load_triple()\n",
    "e2idx, p2idx, params = build_vocab(triples_tr)\n",
    "\n",
    "model = tf.estimator.Estimator(model_fn,\n",
    "                               params = params)\n",
    "\n",
    "for _ in range(Config.n_epochs):\n",
    "    model.train(train_input_fn(triples_tr,\n",
    "                               e2idx,\n",
    "                               p2idx,\n",
    "                               random_state,\n",
    "                               params))\n",
    "evaluate_rank(model,\n",
    "              triples_va,\n",
    "              triples_te,\n",
    "              triples_all,\n",
    "              e2idx,\n",
    "              p2idx,\n",
    "              params['e_vocab_size'],\n",
    "              batch_size = 10*Config.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
